{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Python 创建神经网络模型解决 MNIST \n",
    "\n",
    "numpy 构建神经网络及随机梯度算法训练模型  \n",
    "\n",
    "神经网络与深度学习  \n",
    "Neural Networks and Deep Learning  \n",
    "http://neuralnetworksanddeeplearning.com/index.html  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Platform : win32 [win32/linux]\n",
      "Systerm  : 3.6.10 (default, Mar  5 2020, 10:17:47) [MSC v.1900 64 bit (AMD64)] \n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "import sys\n",
    "print('Platform : {} [win32/linux]'.format(sys.platform))  # 当前平台信息 \n",
    "print('Systerm  : {} '.format(sys.version))  # python 版本"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取训练数据\n",
    "def get_data(pickle_path, p_flag=False):\n",
    "    \"\"\"\n",
    "    :param pickle_path:  python3 pickle 文件路径\n",
    "    :param p_flag:\n",
    "    :return:\n",
    "            X shape:(784, 1)  Y shape:(10, 1)\n",
    "            [(x1, y1),(x2, y2),(x3, y3) ......]\n",
    "    \"\"\"\n",
    "    with open(pickle_path, 'rb') as f:\n",
    "        train_data, test_data = pickle.load(f)\n",
    "\n",
    "    if p_flag:\n",
    "        print('Train data Num:{}'.format(len(train_data)))\n",
    "        print('Test  data Num:{}'.format(len(test_data)))\n",
    "        x, y = train_data[0]\n",
    "        print('X shape:{}  Y shape:{}'.format(x.shape, y.shape))\n",
    "\n",
    "    return train_data, test_data\n",
    "\n",
    "\n",
    "# # mnist 数据读取\n",
    "# def get_data():\n",
    "#     from tensorflow import keras\n",
    "#     mnist = keras.datasets.mnist\n",
    "\n",
    "#     num_classes = 10\n",
    "\n",
    "#     # # the data, split between train and test sets\n",
    "#     mnist_path = os.path.abspath(r'D:\\TFKerasStudyNotes\\Data\\mnist.npz')\n",
    "#     (x_train, y_train), (x_test, y_test) = mnist.load_data(mnist_path)  # 加载MNIST 数据集\n",
    "\n",
    "#     x_train = x_train.reshape(60000, 784)\n",
    "#     x_test = x_test.reshape(10000, 784)\n",
    "#     x_train = x_train.astype('float32')\n",
    "#     x_test = x_test.astype('float32')\n",
    "#     x_train /= 255   # 归一化\n",
    "#     x_test /= 255    # 归一化\n",
    "#     print(x_train.shape[0], 'train samples')\n",
    "#     print(x_test.shape[0], 'test samples')\n",
    "\n",
    "#     # convert class vectors to binary class matrices\n",
    "#     y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "#     y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "#     print('X shape: {}   \\nY Shape: {}'.format(x_train.shape, y_train.shape))\n",
    "\n",
    "#     train_data = [(x.reshape([-1, 1]), y.reshape([-1, 1])) for x, y in zip(x_train, y_train)]\n",
    "#     test_data = [(x.reshape([-1, 1]), y.reshape([-1, 1])) for x, y in zip(x_test, y_test)]\n",
    "\n",
    "#     return train_data, test_data\n",
    "\n",
    "#     # import pickle\n",
    "#     # with open('mnist.pickle', 'wb') as f:\n",
    "#     #     pickle.dump((train_data, test_data), f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 网络模型及前后向计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(object):\n",
    "    \"\"\"\n",
    "    A module to implement the stochastic gradient descent learning\n",
    "    algorithm for a feedforward neural network.  Gradients are calculated\n",
    "    using backpropagation.  Note that I have focused on making the code\n",
    "    simple, easily readable, and easily modifiable.  It is not optimized,\n",
    "    and omits many desirable features.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, sizes):\n",
    "        \"\"\"The list ``sizes`` contains the number of neurons in the\n",
    "        respective layers of the network.  For example, if the list\n",
    "        was [2, 3, 1] then it would be a three-layer network, with the\n",
    "        first layer containing 2 neurons, the second layer 3 neurons,\n",
    "        and the third layer 1 neuron.  The biases and weights for the\n",
    "        network are initialized randomly, using a Gaussian\n",
    "        distribution with mean 0, and variance 1.  Note that the first\n",
    "        layer is assumed to be an input layer, and by convention we\n",
    "        won't set any biases for those neurons, since biases are only\n",
    "        ever used in computing the outputs from later layers.\"\"\"\n",
    "        self.num_layers = len(sizes)\n",
    "        self.sizes = sizes\n",
    "        self.biases = [np.random.randn(y, 1) for y in sizes[1:]]\n",
    "        self.weights = [np.random.randn(y, x)\n",
    "                        for x, y in zip(sizes[:-1], sizes[1:])]\n",
    "\n",
    "    def feedforward(self, a):\n",
    "        \"\"\"Return the output of the network if ``a`` is input.\"\"\"\n",
    "        for b, w in zip(self.biases, self.weights):\n",
    "            a = sigmoid(np.dot(w, a) + b)\n",
    "        return a\n",
    "\n",
    "    def SGD(self, training_data, epochs, mini_batch_size, eta,\n",
    "            test_data=None):\n",
    "        \"\"\"Train the neural network using mini-batch stochastic\n",
    "        gradient descent.  The ``training_data`` is a list of tuples\n",
    "        ``(x, y)`` representing the training inputs and the desired\n",
    "        outputs.  The other non-optional parameters are\n",
    "        self-explanatory.  If ``test_data`` is provided then the\n",
    "        network will be evaluated against the test data after each\n",
    "        epoch, and partial progress printed out.  This is useful for\n",
    "        tracking progress, but slows things down substantially.\"\"\"\n",
    "        if test_data:\n",
    "            n_test = len(test_data)\n",
    "        else:\n",
    "            n_test = None\n",
    "\n",
    "        n = len(training_data)\n",
    "        for j in range(epochs):\n",
    "            random.shuffle(training_data)\n",
    "            mini_batches = [\n",
    "                training_data[k:k + mini_batch_size]\n",
    "                for k in range(0, n, mini_batch_size)]\n",
    "            for mini_batch in mini_batches:\n",
    "                self.update_mini_batch(mini_batch, eta)\n",
    "            if test_data:\n",
    "                print(\"Epoch {0}: {1} / {2}\".format(\n",
    "                    j, self.evaluate(test_data), n_test))\n",
    "            else:\n",
    "                print(\"Epoch {0} complete\".format(j))\n",
    "\n",
    "    def update_mini_batch(self, mini_batch, eta):\n",
    "        \"\"\"Update the network's weights and biases by applying\n",
    "        gradient descent using backpropagation to a single mini batch.\n",
    "        The ``mini_batch`` is a list of tuples ``(x, y)``, and ``eta``\n",
    "        is the learning rate.\"\"\"\n",
    "        nabla_b = [np.zeros(b.shape) for b in self.biases]\n",
    "        nabla_w = [np.zeros(w.shape) for w in self.weights]\n",
    "        for x, y in mini_batch:\n",
    "            # 每个训练数据都进行计算\n",
    "            delta_nabla_b, delta_nabla_w = self.backprop(x, y)\n",
    "            # 偏差值取平均值\n",
    "            nabla_b = [nb + dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]\n",
    "            nabla_w = [nw + dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]\n",
    "        self.weights = [w - (eta / len(mini_batch)) * nw\n",
    "                        for w, nw in zip(self.weights, nabla_w)]\n",
    "        self.biases = [b - (eta / len(mini_batch)) * nb\n",
    "                       for b, nb in zip(self.biases, nabla_b)]\n",
    "\n",
    "    def backprop(self, x, y):\n",
    "        \"\"\"Return a tuple ``(nabla_b, nabla_w)`` representing the\n",
    "        gradient for the cost function C_x.  ``nabla_b`` and\n",
    "        ``nabla_w`` are layer-by-layer lists of numpy arrays, similar\n",
    "        to ``self.biases`` and ``self.weights``.\"\"\"\n",
    "        nabla_b = [np.zeros(b.shape) for b in self.biases]\n",
    "        nabla_w = [np.zeros(w.shape) for w in self.weights]\n",
    "        # feedforward\n",
    "        activation = x\n",
    "        activations = [x]  # list to store all the activations, layer by layer\n",
    "        zs = []  # list to store all the z vectors, layer by layer\n",
    "        for b, w in zip(self.biases, self.weights):\n",
    "            z = np.dot(w, activation) + b\n",
    "            zs.append(z)\n",
    "            activation = sigmoid(z)\n",
    "            activations.append(activation)\n",
    "        # backward pass\n",
    "        delta = self.cost_derivative(activations[-1], y) * sigmoid_prime(zs[-1])\n",
    "        nabla_b[-1] = delta\n",
    "        nabla_w[-1] = np.dot(delta, activations[-2].transpose())\n",
    "        # Note that the variable l in the loop below is used a little\n",
    "        # differently to the notation in Chapter 2 of the book.  Here,\n",
    "        # l = 1 means the last layer of neurons, l = 2 is the\n",
    "        # second-last layer, and so on.  It's a renumbering of the\n",
    "        # scheme in the book, used here to take advantage of the fact\n",
    "        # that Python can use negative indices in lists.\n",
    "        for l in range(2, self.num_layers):\n",
    "            z = zs[-l]\n",
    "            sp = sigmoid_prime(z)\n",
    "            delta = np.dot(self.weights[-l + 1].transpose(), delta) * sp\n",
    "            nabla_b[-l] = delta\n",
    "            nabla_w[-l] = np.dot(delta, activations[-l - 1].transpose())\n",
    "        return nabla_b, nabla_w\n",
    "\n",
    "    def evaluate(self, test_data):\n",
    "        \"\"\"Return the number of test inputs for which the neural\n",
    "        network outputs the correct result. Note that the neural\n",
    "        network's output is assumed to be the index of whichever\n",
    "        neuron in the final layer has the highest activation.\"\"\"\n",
    "        test_results = [(np.argmax(self.feedforward(x)), np.argmax(y))\n",
    "                        for (x, y) in test_data]\n",
    "        return sum(int(x == y) for (x, y) in test_results)\n",
    "\n",
    "    def cost_derivative(self, output_activations, y):\n",
    "        \"\"\"Return the vector of partial derivatives \\partial C_x /\n",
    "        \\partial a for the output activations.\"\"\"\n",
    "        return output_activations - y\n",
    "\n",
    "\n",
    "# 激活函数\n",
    "def sigmoid(z):\n",
    "    \"\"\"The sigmoid function.\"\"\"\n",
    "    return 1.0 / (1.0 + np.exp(-z))\n",
    "\n",
    "\n",
    "# 激活函数 导数\n",
    "def sigmoid_prime(z):\n",
    "    \"\"\"Derivative of the sigmoid function.\"\"\"\n",
    "    return sigmoid(z) * (1 - sigmoid(z))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 创建及训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data Num:60000\n",
      "Test  data Num:10000\n",
      "X shape:(784, 1)  Y shape:(10, 1)\n",
      "Epoch 0: 8905 / 10000\n",
      "Epoch 1: 9093 / 10000\n",
      "Epoch 2: 9210 / 10000\n"
     ]
    }
   ],
   "source": [
    "# 读取数据\n",
    "train_data, test_data = get_data('mnist.pickle', p_flag=True)\n",
    "\n",
    "# 创建模型\n",
    "net = Network([784, 30, 10])\n",
    "# 训练模型\n",
    "net.SGD(train_data, epochs=3, mini_batch_size=30, eta=3.0, test_data=test_data)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: 4817 / 10000\n",
      "Epoch 1: 6235 / 10000\n",
      "Epoch 2: 7186 / 10000\n"
     ]
    }
   ],
   "source": [
    "# 创建模型\n",
    "net = Network([784, 30, 10])\n",
    "# 训练模型\n",
    "net.SGD(train_data, epochs=3, mini_batch_size=30, eta=1.0, test_data=test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: 8821 / 10000\n",
      "Epoch 1: 9077 / 10000\n",
      "Epoch 2: 9154 / 10000\n",
      "Epoch 3: 9207 / 10000\n",
      "Epoch 4: 9233 / 10000\n"
     ]
    }
   ],
   "source": [
    "# 创建模型\n",
    "net = Network([784, 30, 10])\n",
    "# 训练模型\n",
    "net.SGD(train_data, epochs=5, mini_batch_size=50, eta=5.0, test_data=test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 0.5610732 ],\n",
       "        [-0.08921274],\n",
       "        [ 0.55863553],\n",
       "        [ 0.82655925],\n",
       "        [ 0.86223571],\n",
       "        [-0.16102808],\n",
       "        [ 0.65505396],\n",
       "        [ 0.26190059],\n",
       "        [ 0.96006977],\n",
       "        [-0.83747896],\n",
       "        [ 1.50781226],\n",
       "        [ 0.74137992],\n",
       "        [-1.85789504],\n",
       "        [ 1.40978693],\n",
       "        [ 0.8462852 ],\n",
       "        [-0.63133019],\n",
       "        [-0.63105505],\n",
       "        [ 0.56474157],\n",
       "        [ 1.28191445],\n",
       "        [-0.03145859],\n",
       "        [ 0.47289401],\n",
       "        [ 3.04948917],\n",
       "        [-0.00593756],\n",
       "        [-0.59468402],\n",
       "        [ 0.72898537],\n",
       "        [ 0.80183237],\n",
       "        [-0.5224101 ],\n",
       "        [ 2.29873927],\n",
       "        [ 2.12771227],\n",
       "        [-2.03931755]]),\n",
       " array([[-0.8973264 ],\n",
       "        [-0.88358268],\n",
       "        [-1.94794399],\n",
       "        [-2.14912974],\n",
       "        [-0.87194318],\n",
       "        [-0.34064911],\n",
       "        [-1.39945715],\n",
       "        [-3.16183109],\n",
       "        [-2.08068255],\n",
       "        [-0.52336026]])]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.biases\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PY36-tf12",
   "language": "python",
   "name": "py36_b"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
